const cheerio = require('cheerio');
const http = require('http');
const iconv = require('iconv-lite');
const fs = require('fs');
// 引入数据库文件
const db = require("./db");

// 定义一个容器接收
let spiderResult = {
  jsContent: []
};

/**
 * 定义一个函数,传递当前页面进去
 * @param {*} page 
 */
const pageSpider = (page) => {
  let url = page === 1 ? 'http://web.jobbole.com/category/javascript-2' : `http://web.jobbole.com/category/javascript-2/page/${page}/`;
  return new Promise((resolve, reject) => {
    http.get(url, (sres) => {
      var chunks = [];
      let resultArray = [];
      sres.on('data', function (chunk) {
        chunks.push(chunk);
      });
      sres.on('end', function () {
        // 设置编码
        var html = iconv.decode(Buffer.concat(chunks), 'utf-8');
        // 使用类似jq的选择器
        var $ = cheerio.load(html, { decodeEntities: false });
        $('#archive > .floated-thumb').each((idx, element) => {
          imgUrl = $(element).find('.post-thumb > a > img').attr('src');
          title = $(element).find('.post-meta > p:nth-child(1) > a.archive-title').attr('title');
          // 追加到数组中
          resultArray.push({
            imgUrl: imgUrl,
            title: title
          });
          // 插入到数据库
          db('insert into new(title, img_url) values(?,?)', [title, imgUrl], (err, data) => {
            if (err) {
              console.log('错误了');
              return false;
            }
            if (data) {
              console.log('成功插入到mysql中');
            }
          });
          resolve(resultArray);
        });
      });
    });
  });
};


for(let i = 1; i < 10; i++) {
  pageSpider(i).then(data => {
    fs.writeFile('./22.json', JSON.stringify(data, null, 2), { 'flag': 'a+' }, (error) => {
      if (error) {
        console.error('错误');
      }
      console.info('成功');
    });
  });
}

